package com.force.langchain4j.filesource;

import org.apache.commons.io.ByteOrderMark;
import org.apache.tika.detect.AutoDetectReader;
import org.apache.tika.exception.TikaException;
import org.springframework.ai.document.Document;
import org.springframework.ai.reader.TextReader;
import org.springframework.core.annotation.Order;

import java.io.*;
import java.util.ArrayList;
import java.util.List;

@Order(1)
public class TxtFileLoader extends WordFileLoader implements FileSourceLoader{

	private static final  String fileType="txt";
	@Override
	public Object getSourceContent(FileBaseLoader file) {
		String suffix = file.getSuffix();
		InputStream inputStream = file.getInputStream();
		if (fileType.equals(suffix)) {
			try (AutoDetectReader reader = new AutoDetectReader(inputStream)) { // Default encoding if no BOM is found or cannot be determined
				String line;
				int linesPerDoc = 3000;
				List<String> buffer = new ArrayList<>();
				List<Document> documents = new ArrayList<>();

				while ((line = reader.readLine()) != null) {
					buffer.add(line);
					if (buffer.size() == linesPerDoc) {
						documents.add(new Document(String.join("\n", buffer)));
						buffer.clear();
					}
				}

// 最后剩余内容
				if (!buffer.isEmpty()) {
					documents.add(new Document(String.join("\n", buffer)));
				}
				return documents;
			} catch (IOException e) {
				throw new RuntimeException(e);
			} catch (TikaException e) {
				throw new RuntimeException(e);
			}
		}else{
			return super.getSourceContent(file);
		}
	}
}
